{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b5161745",
   "metadata": {},
   "source": [
    "# <center>Assignment 3, Jane Doe, April 18, 2021<center>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f466dd2f",
   "metadata": {},
   "source": [
    "## Question 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "68c87998",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Duration = 1131.0478338\n",
      "Duration = 125.35033570000019\n",
      "one vs one strategy svc C=5, gamma=0.05 accuracy is:  0.9539124668435013\n",
      "one vs rest strategy svc C=5, gamma=0.05 accuracy is:  0.9542440318302388\n",
      "one vs rest strategy svc C=10, gamma=0.00 kernel=rbf accuracy is:  0.9880636604774535\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "import time\n",
    "import numpy as np\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "def tic():\n",
    "    global __start_interval \n",
    "    __start_interval = time.perf_counter()\n",
    "def toc():\n",
    "    global __start_interval\n",
    "    print(f\"Duration = {time.perf_counter() - __start_interval}\")\n",
    "    mnist = fetch_openml('mnist_784', version=1, as_frame=False)\n",
    "mnist = fetch_openml('mnist_784', version=1, as_frame=False)\n",
    "X, y = mnist[\"data\"], mnist[\"target\"]\n",
    "y = y.astype(np.uint8)\n",
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]\n",
    "scaling = StandardScaler()\n",
    "X_train = scaling.fit_transform(X_train)\n",
    "X_test = scaling.fit_transform(X_test)\n",
    "idx = (y_train==2)|(y_train == 3)|(y_train ==8)# select out digit 2, 3, 8 that will save a lot of time\n",
    "x = X_train[idx]\n",
    "y = y_train[idx]\n",
    "idx_ = (y_test==2)|(y_test == 3)|(y_test ==8)\n",
    "x_ = X_test[idx_]\n",
    "y_ = y_test[idx_]\n",
    "tic()\n",
    "svc = OneVsRestClassifier(SVC(kernel='linear')).fit(x,y)\n",
    "toc()\n",
    "tic()\n",
    "svc_one = OneVsOneClassifier(SVC(kernel='linear')).fit(x,y)\n",
    "toc()\n",
    "y_pred = svc.predict(x_)\n",
    "y_pred_one = svc_one.predict(x_)\n",
    "print(\"one vs one strategy svc C=5, gamma=0.05 accuracy is: \", accuracy_score(y_, y_pred_one))\n",
    "print(\"one vs rest strategy svc C=5, gamma=0.05 accuracy is: \", accuracy_score(y_, y_pred))\n",
    "svc_tune = OneVsRestClassifier(SVC(C=10, gamma=0.001, kernel=\"rbf\")).fit(x,y)\n",
    "y_tune = svc_tune.predict(x_)\n",
    "print(\"one vs rest strategy svc C=10, gamma=0.00 kernel=rbf accuracy is: \", accuracy_score(y_, y_tune))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56e0c827",
   "metadata": {},
   "source": [
    "## The result show that one vs rest strategy takes more time than one vs one strategy under the condition of use SVM, and the accuracy of both are almost the same. Tuning the paramters C=10, gamma=0.00 kernel=rbf achives a curracy at 0.98"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0a79f0e",
   "metadata": {},
   "source": [
    "## Question 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0b506284",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5c6ea9b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_reg = LinearSVR(epsilon=1.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a990eaad",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "34194610",
   "metadata": {},
   "outputs": [],
   "source": [
    "housing = fetch_california_housing()#load dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d4987e70",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(housing['data'],housing['target'], test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "806762a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaling = StandardScaler()# data scaling\n",
    "X_train = scaling.fit_transform(X_train)\n",
    "X_test = scaling.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cf67f2f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.326196  ,  0.34849025, -0.17491646, ...,  0.05137609,\n",
       "        -1.3728112 ,  1.27258656],\n",
       "       [-0.03584338,  1.61811813, -0.40283542, ..., -0.11736222,\n",
       "        -0.87669601,  0.70916212],\n",
       "       [ 0.14470145, -1.95271028,  0.08821601, ..., -0.03227969,\n",
       "        -0.46014647, -0.44760309],\n",
       "       ...,\n",
       "       [-0.49697313,  0.58654547, -0.60675918, ...,  0.02030568,\n",
       "        -0.75500738,  0.59946887],\n",
       "       [ 0.96545045, -1.07984112,  0.40217517, ...,  0.00707608,\n",
       "         0.90651045, -1.18553953],\n",
       "       [-0.68544764,  1.85617335, -0.85144571, ..., -0.08535429,\n",
       "         0.99543676, -1.41489815]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ae819e3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\env\\ad\\lib\\site-packages\\sklearn\\svm\\_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearSVR(epsilon=1.5, random_state=42)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_reg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3e052257",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = svm_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f2e6f0b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mse:  0.7994035077638033\n"
     ]
    }
   ],
   "source": [
    "print(\"mse: \", mean_squared_error(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aefe584b",
   "metadata": {},
   "source": [
    "## Question 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c6705d7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3cdf8346",
   "metadata": {},
   "outputs": [],
   "source": [
    "Xm, ym = make_moons(n_samples=10000, noise=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "e8d9e594",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(Xm,ym, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e88753a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=DecisionTreeClassifier(),\n",
       "             param_grid={'max_leaf_nodes': [2, 3, 4, 5, 6, 7, 8]})"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt = DecisionTreeClassifier()\n",
    "paramters = {\"max_leaf_nodes\":[2,3,4,5,6,7,8]}\n",
    "clf = GridSearchCV(dt, paramters)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "cc2ac8c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_leaf_nodes': 4}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "9f7dd2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_nodes = clf.best_params_['max_leaf_nodes']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "26cc1625",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt = DecisionTreeClassifier(max_leaf_nodes=best_nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "f2833f4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(max_leaf_nodes=4)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "82cea7b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = dt.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "16773e19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.855"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bb3aace",
   "metadata": {},
   "source": [
    "## Question 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "50435085",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.multiclass import OneVsOneClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "00127801",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = fetch_openml('mnist_784', version=1, as_frame=False)\n",
    "X, y = mnist[\"data\"], mnist[\"target\"]\n",
    "X_train, X_test, X_val, y_train, y_test, y_val = X[:50000], X[50000:60000], X[60000:], y[:50000], y[50000:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d108fa3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc = OneVsOneClassifier(SVC(kernel='linear'))\n",
    "rnd_clf = RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "et_clf = ExtraTreesClassifier(n_estimators=100, random_state=42)\n",
    "voting_clf = VotingClassifier(\n",
    "    estimators=[('et_clf', et_clf), ('rf', rnd_clf), ('svc', svc)],\n",
    "    voting='hard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f5dedc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExtraTreesClassifier 0.9703\n",
      "RandomForestClassifier 0.968\n"
     ]
    }
   ],
   "source": [
    "for clf in (et_clf, rnd_clf, svc, voting_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_val)\n",
    "    print(clf.__class__.__name__, accuracy_score(y_val, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa84f151",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_ = voting_clf.predict(X_test)\n",
    "print(voting_clf.__class__.__name__, accuracy_score(y_test, y_pred_))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
